# -*- coding: utf-8 -*-

import os
import re
import sys
reload(sys)
sys.setdefaultencoding('utf8')

from lxml import etree
from lxml import html as HTML 
from lxml.html import *

from bs4 import BeautifulSoup
from bs4 import Comment 
from bs4 import Declaration 
from bs4 import Doctype 
from bs4 import NavigableString 

# < single tag > remove
# < common tag > value </ common tag> extract value then put it to next loop
# < script > </script> remove

class PageCleaner(object):
    single_tag_list = ['meta', 'link', 'style', 'img', 'input', 'hr', 'br', '!DOCTYPE', '!--']
    exculded_tag_list = ['script', 'img', 'button']

    def __init__(self):
        self.html = ''
        self.node_recorder = list()
        self.extract_content = '' 

    def feed_html(self, filename=''):
        with open(filename, 'r') as f:
            self.html = f.read()

    def clean(self):
        #html = HTML.fromstring(self.html)
        soup = BeautifulSoup(self.html)
        body_content = ""

        #print soup.contents
        #print soup.children

        self.dsf(soup) 

        #self.extract_content = body_content
        #print self.extract_content

    def dsf(self, tag):
        for child in tag.children:
            if isinstance(child, Comment):
                #print 'comment'
                continue
            if isinstance(child, Doctype):
                #print 'Doctype'
                continue
            if isinstance(child, Declaration):
                #print 'Declaration'
                continue

            if isinstance(child, NavigableString):
                if len(child) > 4:
                    #print len(child)
                    #print "[%s]" % child
                    self.extract_content += child + '\n'
                continue

            #print child.name
            if child.name in self.single_tag_list:
                continue
            #print child.string

            if child.name in self.exculded_tag_list:
                #print child.string
                continue

            self.dsf(child)


    def get_content(self):
        return self.html

    def output_file(self, filename):
        with open(filename, 'a+') as f:
            f.write(self.extract_content)

import sys, os
if __name__ == '__main__':
    cleaner = PageCleaner()
    #cleaner.feed_html(sys.argv[1])
    cleaner.feed_html('./data/36kr.com')
    cleaner.clean()
    cleaner.output_file('example.txt')

    #dirlist = os.listdir('./data/');
    #for i in dirlist:
    #    cleaner.feed_html(os.path.join(os.getcwd(), 'data', i))
    #    cleaner.clean()
    #with open(os.path.join('data_out', i+'.txt'), 'a') as f:
    #    f.write(cleaner.extract_content)
